{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "04ecc3ce",
   "metadata": {},
   "source": [
    "![](./pics/attention.png)\n",
    "\n",
    "![](./pics/attention2.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08330618",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a784b190",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Attn(nn.Module):\n",
    "    def __init__(self, query_size, key_size, value_size1, value_size2, output_size):\n",
    "        '''\n",
    "        query_size代表query的最后一维大小\n",
    "        key_size代表key的最后一维大小\n",
    "        value_size1代表value的倒数第二维大小\n",
    "        value = (1, value_size1, value_size2)\n",
    "        output_size代表输出的最后一维大小\n",
    "        '''\n",
    "        super(Attn, self).__init__()\n",
    "        self.query_size = query_size\n",
    "        self.key_size = key_size\n",
    "        self.value_size1 = value_size1\n",
    "        self.value_size2 = value_size2\n",
    "        self.output_size = output_size\n",
    "        \n",
    "        # 初始化注意力机制实现需要第一步中需要的线性层\n",
    "        self.attn = nn.Linear(self.query_size+self.key_size, value_size1)\n",
    "        \n",
    "        # 初始化注意力机制实现第三步中需要的线性层\n",
    "        self.attn_combine = nn.Linear(self.query_size+self.value_size2, output_size)\n",
    "        \n",
    "    def forward(self, Q, K, V):\n",
    "        '''\n",
    "        Q, K, V都是三维张量\n",
    "        '''\n",
    "        # 将Q， K进行纵轴拼接，做一次线性变换，最后使用softmax处理获得结果\n",
    "        attn_weights = F.softmax(self.attn(torch.cat((Q[0], K[0]), 1)), dim=1)\n",
    "        \n",
    "        # 当二者都是三维张量且第一维代表batch条数时，做bmm运算\n",
    "        attn_applied = torch.bmm(attn_weights.unsqueeze(0), V)\n",
    "        \n",
    "        # 计算输出\n",
    "        output = torch.cat((Q[0], attn_applied[0]), 1)\n",
    "        output = self.attn_combine(output).unsqueeze(0)\n",
    "        return output, attn_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed31e20b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 调用\n",
    "query_size = 32\n",
    "key_size = 32\n",
    "value_size1 = 32\n",
    "value_size2 = 64\n",
    "output_size = 64\n",
    "\n",
    "attn = Attn(query_size, key_size, value_size1, value_size2, output_size)\n",
    "Q = torch.randn(1,1,32)\n",
    "K = torch.randn(1,1,32)\n",
    "V = torch.randn(1,32,64)\n",
    "output = attn(Q, K, V)\n",
    "print(output[0])\n",
    "print(output[0].size())\n",
    "print(output[1])\n",
    "print(output[1].size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd3f61dd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torchX",
   "language": "python",
   "name": "torchx"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
